import requests
from my_fake_useragent import UserAgent
import pandas as pd
import numpy as py
import pymongo
import os
import time
import random
from pyecharts.charts import Bar

client = pymongo.MongoClient('localhost',27017)
mydb = client['travel']
test = mydb['guoqing']
ua = UserAgent()
PLACE_EXCEL_PATH = 'qunar.xlsx'

def spider_place(keyword, page):
    url = 'http://piao.qunar.com/ticket/list.json?keyword={}&region=&from=mpl_search_suggest&page={}'.format(keyword, page)
    headers = {'User-Agent': ua.random()}
    response = requests.get(url,headers = headers)
    res_list = get_spider_info(response.json())
    save_excel(res_list)


def get_spider_info(res_json):
    sight_list = res_json['data']['sightList']
    place_list = []
    for sight in sight_list:
        goods = {
            'id': sight['sightId'],
            'name': sight['sightName'],
            'star': sight.get('star','无'),
            'score': sight.get('score', 0),
            'price': sight.get('qunarPrice', 0),
            'sale':sight.get('saleCount', 0),
            'districts': sight['districts'],
            'point':sight['point'],
            'intro': sight.get('intro', '')
        }
        place_list.append(goods)
        test.insert_one(goods)
    return place_list

def save_excel(place_list):
    if os.path.exists(PLACE_EXCEL_PATH):
        df = pd.read_excel(PLACE_EXCEL_PATH)
        df = df.append(place_list)
    else:
        df = pd.DataFrame(place_list)
    writer = pd.ExcelWriter(PLACE_EXCEL_PATH)
    df.to_excel(excel_writer=writer, columns=['id', 'name', 'star', 'score', 'price', 'sale', 'districts', 'point', 'intro'], index = False, encoding='utf-8', sheet_name= '去哪儿国庆热门')
    writer.save()
    writer.close()

def patch_spider_place(keyword):
    if os.path.exists(PLACE_EXCEL_PATH):
        os.remove(PLACE_EXCEL_PATH)
    for i in range(1,36):
        print(f'正在爬取{keyword}第{i}页')
        spider_place(keyword, i)
        time.sleep(random.randint(1,3))
    print('爬取完成')

if __name__ == '__main__':
    patch_spider_place('国庆旅游景点')
    # spider_place('国庆旅游景点', 1)